#!/usr/bin/env python
#coding=utf8
"""
@描述：将看准数据load到我的本机上900w条
      采用mysqldump后source的方式太慢
@日期：2015年01月31日 星期六
@作者：hongxingfan
"""

from DBUtils import MySQLDB
import sys
from DBArgs import *

# 连接配置,与业务无关的代码不要放在main中

if __name__ == "__main__":

    # local数据库连接
    DB_127 = MySQLDB(DB127.host, DB127.user, DB127.passwd, DB127.port, DB127.charset)
    insert = """
            INSERT INTO fetched.jobs(id,url,provider,title,company,amount,
            place,address_id,experience,
            salary,degree,kind,age,sex,jobclass,industry,publish_date,update_date,
            welfare,extra,description,requirement,STATUS,_update_at) 
            VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
        """
    # select出来DB251中的数据
    select = """
            SELECT id,url,provider,title,company,amount,place,address_id,experience,
            salary,degree,kind,age,sex,jobclass,industry,publish_date,update_date,
            welfare,extra,description,requirement,status,_update_at FROM jobs.jobs 
            WHERE id<%s ORDER BY id DESC LIMIT 0,500
        """
    DB_251 = MySQLDB(DB251.host, DB251.user, DB251.passwd, DB251.port, DB251.charset)

    selectParam = [sys.maxint]
    # 输入过程中会中断(记录内容过大),增加一个参数继续执行
    if(len(sys.argv) == 2):
        selectParam[0] = sys.argv[1]
    values = []
    total = 0
    while(True):
        selectResult = DB_251.select(select, selectParam)
        count = 0
        del(values[:]) # 清空values
        for r in selectResult:
            values.append(r)
            count += 1
            total += 1
            if(r[0] < selectParam[0]):
                selectParam[0] = r[0]
        # 刷新到数据库中
        DB_127.insert(insert, tuple(values))
        print("刷新到数据库中 %d 条记录,一共刷新 %d 条记录!" %(count, total))
        if(count < 10):
            break
    DB_127.close()
    DB_251.close()
